# coding=utf-8
import codecs
import fnmatch
import json
import sys

import os

from core.m_logging import logger
from spiders import amazonAnsiListingCoreSpider
from spiders.nodeSpider import NodeSpider

reload(sys)
sys.setdefaultencoding('utf8')

if len(sys.argv) >= 4:
    task_no = sys.argv[1]
    task_dir = sys.argv[2]
    batch_no = sys.argv[3]
else:
    task_no = "default_task"
    task_dir = "../tasks"
    batch_no = "default_batch"

node_no_list = []
logger.info("task_dir is %s..." % task_dir)

task_files = fnmatch.filter(os.listdir(task_dir), 'task*.json')
logger.info("loading tasks %s" % task_files)
for f in task_files:
    with codecs.open("%s/%s" % (task_dir, f), mode='r', encoding='utf-8') as opened_file:
        r = json.load(opened_file)
        node_no_list += list(r["list"])

# set module variable after import
amazonAnsiListingCoreSpider.task_state_dir = task_dir
amazonAnsiListingCoreSpider.batch_no = batch_no
amazonAnsiListingCoreSpider.task_no = task_no

nodeSpider = NodeSpider(batch_no, task_no)
for current_node_no in node_no_list:
    amazonAnsiListingCoreSpider.scrape(current_node_no, [])  # [] means populate all pages and go through all of them

logger.info("process is completed...")